#/bin/bash
Cur_day=`date -d '1 day ago' +%Y%m%d`
TIME_WAIT_MAX=36000
TIME_SLEEP=300
#运行时传城市参数进来 $1
bj_today=$1"_$Cur_day"
Local_inputdata_path=/usr/local/apps/hadoop/data
Local_code_path=/usr/local/apps/hadoop/hezu
Local_hadoop_path=/usr/local/apps/hadoop-2.7.2/share/hadoop/tools/lib
#删除文件，如果存在的话
if [ -f /usr/local/apps/hadoop/data/house_hezu_gov.gz ]
      then
      rm -rf /usr/local/apps/hadoop/data/house_hezu_gov.gz
fi
#运行时传该城市对应的数据库名作为第二个参数
python /usr/local/apps/hadoop/hezu/dumpdata.py $2
#compress the data, format=gzip
gzip $Local_inputdata_path/house_hezu_gov
hadoop fs -mkdir /house_hezu/$bj_today
hadoop fs -copyFromLocal $Local_inputdata_path/house_hezu_gov.gz /house_hezu/$bj_today/house_hezu_gov.gz
mkdir $Local_inputdata_path/hezu/$bj_today/
mkdir $Local_inputdata_path/hezu/$bj_today/house_hezu
mkdir $Local_inputdata_path/hezu/$bj_today/house_hezu_gov
mkdir $Local_inputdata_path/hezu/$bj_today/house_hezu_pic
#mapred3
hadoop jar $Local_hadoop_path/hadoop-streaming-2.7.2.jar \
-input /house_hezu/$bj_today/house_hezu_gov.gz \
-output  /house_hezu/$bj_today/mapred3 \
-mapper '/usr/bin/php mapper_3.php' \
-reducer '/usr/bin/php reducer3_1.php' \
-file '/usr/local/apps/hadoop/hezu/mapper_3.php' \
-file '/usr/local/apps/hadoop/hezu/reducer3_1.php'
#mapred4
hadoop jar $Local_hadoop_path/hadoop-streaming-2.7.2.jar \
-D mapreduce.job.reduces=1 \
-input /house_hezu/$bj_today/mapred3/part-00000 \
-output /house_hezu/$bj_today/mapred4 \
-mapper '/usr/bin/php mapper_4.php' \
-reducer '/usr/bin/php reducer_4_3.php' \
-file '/usr/local/apps/hadoop/hezu/mapper_4.php' \
-file '/usr/local/apps/hadoop/hezu/reducer_4_3.php'
#step5 sql file(house_hezu_info)
hadoop jar $Local_hadoop_path/hadoop-streaming-2.7.2.jar \
-D mapreduce.job.reduces=0 \
-input /house_hezu/$bj_today/mapred4/part-00000 \
-output /house_hezu/$bj_today/mapred5 \
-mapper '/usr/bin/php mapper5.php' \
-file '/usr/local/apps/hadoop/hezu/mapper5.php'
hadoop fs -copyToLocal /house_hezu/$bj_today/mapred5/* $Local_inputdata_path/hezu/$bj_today/
#step6 sql file(house_hezu)
hadoop jar $Local_hadoop_path/hadoop-streaming-2.7.2.jar \
-D mapreduce.job.reduces=0 \
-input /house_hezu/$bj_today/mapred4/part-00000 \
-output /house_hezu/$bj_today/mapred6 \
-mapper '/usr/bin/php mapper6.php' \
-file '/usr/local/apps/hadoop/hezu/mapper6.php'
hadoop fs -copyToLocal /house_hezu/$bj_today/mapred6/* $Local_inputdata_path/hezu/$bj_today/house_hezu
#step7 sql file(house_hezu_gov)
hadoop jar $Local_hadoop_path/hadoop-streaming-2.7.2.jar \
-D mapreduce.job.reduces=0 \
-input /house_hezu/$bj_today/mapred4/part-00000 \
-output /house_hezu/$bj_today/mapred7 \
-mapper '/usr/bin/php mapper7_2_20.php' \
-file '/usr/local/apps/hadoop/hezu/mapper7_2_20.php'
hadoop fs -copyToLocal /house_hezu/$bj_today/mapred7/* $Local_inputdata_path/hezu/$bj_today/house_hezu_gov
#step 8(house_hezu_pic)
hadoop jar $Local_hadoop_path/hadoop-streaming-2.7.2.jar \
-D mapreduce.job.reduces=0 \
-input /house_hezu/$bj_today/mapred4/part-00000 \
-output /house_hezu/$bj_today/mapred8 \
-mapper '/usr/bin/php mapper8.php' \
-file '/usr/local/apps/hadoop/hezu/mapper8.php'
#copyToLocal
hadoop fs -copyToLocal /house_hezu/$bj_today/mapred8/* $Local_inputdata_path/hezu/$bj_today/house_hezu_pic
